今天回來Client應用方這裡, 就是應用程式端這裡聊Instrumentation libraries.
畢竟規範了這麼多Model, Interface, Layer design, 就是希望能讓各開發者們可以加入開發其周遭.
OTel其實提供了很多Automatic Instrumentation libraries.
這些Automatic Instrumentation要根據各語言以及框架來選用添加的.
讓我們開發者在使用OpenTelemetry上更是方便,
不寫任何程式碼或幾乎少少的程式碼設定, 就可以達到所需要的三本柱相關的效果.

像.Net有 System.Diagnostics.Activity內建來產生trace或一些監控參考.

Azure與OTel相關文章能參考此處Azure Monitor

但是像Go是沒有支援自動檢測的功能, 就需要依賴於特定的instrumentation libraries生成telemetry data.

想要什麼指標或類型的telemetry data由開發人員決定合理的檢測範圍.

以下列舉一些OTel內建的GO instrumentation libraries以及提供的telemetry data範圍.

3rd Party Go instrumentation

挑otelsql來玩看看

docker-compose.yml

version: "3.7"
services:
  mysql:
    image: mysql:latest
    ports:
      - 3306:3306
    environment:
      - MYSQL_ROOT_PASSWORD=otel_password
      - MYSQL_DATABASE=db

  client:
    build:
      dockerfile: $PWD/Dockerfile
      context: ..
    ports:
      - 2222:2222
    depends_on:
    - mysql
    - otel-collector

  # Jaeger
  jaeger-all-in-one:
    image: jaegertracing/all-in-one:latest
    ports:
      - "16686:16686"
      - "14268"
      - "14250"
      - "6831:6831/udp"

  otel-collector:
    image: otel/opentelemetry-collector-contrib-dev:latest
    command: ["--config=/etc/otel-collector-config.yaml", ""]
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "1888:1888"   # pprof extension
      - "8888:8888"   # Prometheus metrics exposed by the collector
      - "8889:8889"   # Prometheus exporter metrics
      - "13133:13133" # health_check extension
      - "4317:4317"   # OTLP gRPC receiver
      - "55679:55679" # zpages extension
      - "14268:14268"
    depends_on:
      - jaeger-all-in-one

  prometheus:
    container_name: prometheus
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"

otel-collector-config.yaml
這次使用了prometheus_simple, 來抓取target的metric data

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: "0.0.0.0:4317"
  prometheus_simple:
        collection_interval: 10s
        endpoint: "client:2222"

exporters:
  prometheus:
    endpoint: "0.0.0.0:8889"
    const_labels:
      label1: value1

  logging:

  zipkin:
    endpoint: "http://zipkin-all-in-one:9411/api/v2/spans"
    format: proto

  jaeger:
    endpoint: jaeger-all-in-one:14250
    tls:
      insecure: true

processors:
  batch:

extensions:
  health_check:
  pprof:
    endpoint: :1888
  zpages:
    endpoint: :55679

service:
  extensions: [pprof, zpages, health_check]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [logging, zipkin, jaeger]
    metrics:
      receivers: [otlp, prometheus_simple]
      processors: [batch]
      exporters: [logging, prometheus]

prometheus.yaml
這個job, 是抓取OTel collector的metrics

scrape_configs:
  - job_name: 'otel-collector'
    scrape_interval: 10s
    static_configs:
      - targets: ['otel-collector:8889']
      - targets: ['otel-collector:8888']

package main

import (
	"context"
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/XSAM/otelsql"
	_ "github.com/go-sql-driver/mysql"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/exporters/prometheus"
	"go.opentelemetry.io/otel/metric/global"
	controller "go.opentelemetry.io/otel/sdk/metric/controller/basic"
	"go.opentelemetry.io/otel/sdk/metric/export/aggregation"
	processor "go.opentelemetry.io/otel/sdk/metric/processor/basic"
	selector "go.opentelemetry.io/otel/sdk/metric/selector/simple"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

const instrumentationName = "github.com/XSAM/otelsql/example"

var serviceName = semconv.ServiceNameKey.String("otesql-example")

var mysqlDSN = "root:otel_password@tcp(mysql)/db?parseTime=true"

func initTracer() {
	ctx := context.Background()
	ctx, cancel := context.WithTimeout(ctx, time.Second)
	defer cancel()
	conn, err := grpc.DialContext(ctx, "otel-collector:4317", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock())
	if err != nil {
		log.Fatal(err)
	}

	traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn))

	if err != nil {
		log.Fatal(err)
	}
	bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
	tp := sdktrace.NewTracerProvider(
		sdktrace.WithSampler(sdktrace.AlwaysSample()),
		sdktrace.WithSpanProcessor(bsp),
		sdktrace.WithResource(resource.NewWithAttributes(
			semconv.SchemaURL,
			serviceName,
		)),
	)

	otel.SetTracerProvider(tp)
}

func initMeter() {
	c := controller.New(
		processor.NewFactory(
			selector.NewWithHistogramDistribution(),
			aggregation.CumulativeTemporalitySelector(),
			processor.WithMemory(true),
		),
		controller.WithResource(resource.NewWithAttributes(
			semconv.SchemaURL,
			serviceName,
		)),
	)
	metricExporter, err := prometheus.New(prometheus.Config{}, c)
	if err != nil {
		log.Fatalf("failed to install metric exporter, %v", err)
	}
	global.SetMeterProvider(metricExporter.MeterProvider())

	http.HandleFunc("/", metricExporter.ServeHTTP)
	go func() {
		_ = http.ListenAndServe(":2222", nil)
	}()
	fmt.Println("Prometheus server running on :2222")
}

func main() {
	initTracer()
	initMeter()

	// Connect to database
	db, err := otelsql.Open("mysql", mysqlDSN, otelsql.WithAttributes(
		semconv.DBSystemMySQL,
	))
	if err != nil {
		panic(err)
	}
	defer db.Close()

	err = otelsql.RegisterDBStatsMetrics(db, otelsql.WithAttributes(
		semconv.DBSystemMySQL,
	))
	if err != nil {
		panic(err)
	}

	err = run(db)
	if err != nil {
		panic(err)
	}

	fmt.Println("Example finished updating, please visit :2222")

	select {}
}

func run(db *sql.DB) error {
	// Create a parent span (Optional)
	tracer := otel.GetTracerProvider()
	ctx, span := tracer.Tracer(instrumentationName).Start(context.Background(), "example")
	defer span.End()

	err := query(ctx, db)
	if err != nil {
		span.RecordError(err)
		return err
	}
	return nil
}

func query(ctx context.Context, db *sql.DB) error {
	// Make a query
	rows, err := db.QueryContext(ctx, `SELECT CURRENT_TIMESTAMP`)
	if err != nil {
		return err
	}
	defer rows.Close()

	var currentTime time.Time
	for rows.Next() {
		err = rows.Scan(&currentTime)
		if err != nil {
			return err
		}
	}
	fmt.Println(currentTime)
	return nil
}

首先執行

docker-compose up -d

稍待一會後執行client來看看

docker-compose up client

一樣先看看OTel collector的pipeline配置了什麼
http://localhost:55679/debug/pipelinez

然後打開Prometheus UI http://localhost:9090/

可以看到我們在resource context給的service name有出來,
且有如期收到該instrumentation library所吐出的metrics data

最後看看Jaeger UI
http://localhost:16686/

以下就是這次Trace底下的所有Span

今日小心得

Automatic Instrumentation真的方便XD
在公司寫.NetCore專案時, 一些Automatic Instrumentation加入專案後啟用, 很多資訊都被捕捉出來了, 像這次的sql span也有.
這次展示的otelsql, 雖然span的範圍要自己start, 但metric則是設定好後就自己去處理了. 挺方便
還是要看各語言或框架的生態圈怎麼去擴展了.

OpenTelemetry的部份暫時先介紹到這.
希望未來的工作機會上, 有可能把這些可觀測性的價值與文化, 帶入公司與團隊.
讓Dev與Ops能相互合作, 相互味彼此提供價值.
希望能加入這樣的團隊XD
未來也可能在以這主題, 聊的更深, 結合的更廣.

下面提供一些不錯的連結供學習參考

補充的學習資源

awesome-opentelemetry
UpTrace
Aspecto
signoz
A beginner’s guide to OpenTelemetry